#!/usr/bin/env python
# encoding: utf-8
"""
#-------------------------------------------------------------------#
#                   CONFIDENTIAL --- CUSTOM STUDIOS                 #     
#-------------------------------------------------------------------#
#                                                                   #
#                   @Project Name : 全球法律在线项目                #
#                                                                   #
#                   @File Name    : phl_judgov_bilateral.py                      #
#                                                                   #
#                   @Programmer   : 李建                            #
#                                                                   #  
#                   @Start Date   : 2020-10-14 10:08                 #
#                                                                   #
#                   @Last Update  : 2020-10-14 10:08                 #
#                                                                   #
#-------------------------------------------------------------------#
# Classes:Spider:菲律宾最高法院图书馆Bilateral数据采集               #
#                                                                   #
#-------------------------------------------------------------------#
"""
import hashlib

import scrapy
from ..items import MyFileItem
import re


class Spider(scrapy.Spider):
    name = 'phl_judgov_bilateral'
    allowed_domains = ['elibrary.judiciary.gov.ph/thebookshelf']
    # 菲律宾最高法院电子图书馆地址
    start_urls = ['https://elibrary.judiciary.gov.ph/thebookshelf/35']
    id = 0

    def parse(self, response):
        # 获得成员列表
        trs = response.xpath('//*[@id="container_date"]/a')
        for tr in trs:
            # 详细网址
            detailUrlS = tr.xpath('./@href')
            if len(detailUrlS):
                detailUrl = detailUrlS.get().strip()
                yield response.follow(tr, callback=self.mothparse, dont_filter=True)

    def mothparse(self, response):
        # 获得成员列表
        trs = response.xpath('//*[@id="container_title"]/ul/li')
        for tr in trs:
            # 生效日期
            effectiveDateS = tr.xpath('./a/text()')
            effectiveDate = ''
            if len(effectiveDateS):
                for effectiveDateSl in effectiveDateS:
                    effectiveDate = effectiveDate + ' ' + effectiveDateSl.get().strip()
                effectiveDate = re.sub(r'''^ *''', r'''''', effectiveDate)  # 去除开头的空格
            # 双边条约名称
            titleS = tr.xpath('./a/small/text()')
            title = ''
            if len(titleS):
                for titleSl in titleS:
                    title = title + ' ' + titleSl.get().strip()
                title = re.sub(r'''^ *''', r'''''', title)  # 去除开头的空格
            # 详细网址
            detailUrlS = tr.xpath('./a/@href')
            if len(detailUrlS):
                detailUrl = detailUrlS.get().strip()
                yield response.follow(detailUrl, callback=self.detaiparse, dont_filter=True,
                                      meta={'effectiveDate': effectiveDate, 'title': title})

    def detaiparse(self, response):
        detailUrls = response.xpath('//*[@id="left"]/div/div[1]/a/@href')
        metaa = response.meta
        if len(detailUrls):
            detailUrl = detailUrls.get().strip()
            yield response.follow(detailUrl, callback=self.detailed, dont_filter=True, meta=metaa)

    # 解析详情页
    def detailed(self, response):
        item = MyFileItem()
        if response.status == 200:

            # 国别及文件信息
            country = 'Philippines'
            website = 'judiciary'
            modular = 'bilateral'
            title = response.meta['title']
            title = re.sub(r'''"''', r"'", title)  # 修改双引号为单引号
            effectiveDate = response.meta['effectiveDate']
            # 网址
            detailUrl = response.url
            downloadUrl = detailUrl

            # 下载文件名
            fina = ''
            if len(detailUrl) > 0:
                fina = 'f' + str(hashlib.md5(detailUrl.encode('utf-8')).hexdigest())

            # 唯一ID
            systemid = str(hashlib.md5(detailUrl.encode('utf-8')).hexdigest())

            item['file_urls'] = ''
            item['country'] = country
            item['website'] = website
            item['modular'] = modular
            item['ext'] = 'pdf'
            item['fina'] = fina
            item['title'] = title
            item['detail'] = ''
            item['detailUrl'] = detailUrl
            item['downloadUrl'] = ''

            item['Title'] = title
            item['SortA'] = 'LAWCOUNTRYFLB'
            item['StateParty'] = ''
            item['PublishDate'] = effectiveDate
            item['EffectiveDate'] = ''
            item['FullText'] = ''
            item['SortB'] = 'LANGUAGEYY'
            item['SortC'] = ''
            item['Articles'] = ''
            item['Chapter'] = ''
            item['Section'] = ''
            item['SYS_FLD_DIGITFILENAME'] = fina
            item['FileUrl'] = ''
            item['DownLoadUrl'] = downloadUrl
            item['DownLoadWebNameC'] = '菲律宾最高法院电子图书馆'
            item['DownLoadWebNameE'] = "Supreme Court E-Library"
            item['SYSID'] = systemid
            item['Website'] = 'Bilateral'
            item['Isconversion'] = '1'

            yield item

